{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.1 64-bit ('Python3_7_2')"
  },
  "interpreter": {
   "hash": "ceed3ede7d2ae4746b1bde0ed48f83d28ba93d0b68e140a25bb2fbb7cbabeb22"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('butler.txt','r',encoding='utf8') as r:\n",
    "   lines =r.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "12131\n"
     ]
    }
   ],
   "source": [
    "print(len(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [0]*len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "12131"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "len(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'sentence':lines,\"label\":labels},columns=[\"sentence\",\"label\"] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                            sentence  label\n",
       "0  ﻿Sing, O goddess, the anger of Achilles son of...      0\n",
       "1  countless ills upon the Achaeans. Many a brave...      0\n",
       "2  hurrying down to Hades, and many a hero did it...      0\n",
       "3  vultures, for so were the counsels of Jove ful...      0\n",
       "4  which the son of Atreus, king of men, and grea...      0"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>sentence</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>﻿Sing, O goddess, the anger of Achilles son of...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>countless ills upon the Achaeans. Many a brave...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>hurrying down to Hades, and many a hero did it...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>vultures, for so were the counsels of Jove ful...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>which the son of Atreus, king of men, and grea...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(test_list):\n",
    "    t = Tokenizer()\n",
    "    t.fit_on_texts(test_list)\n",
    "    result = t.texts_to_sequences(test_list)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "clines = convert(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "12131\n"
     ]
    }
   ],
   "source": [
    "print(len(clines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[4551, 340, 312, 1, 317, 3, 54, 21, 3, 173, 12, 224]"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "clines[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[4551, 340, 312, 1, 317, 3, 54, 21, 3, 173, 12, 224]\n[1792, 3376, 45, 1, 44, 108, 9, 179, 623, 47, 17, 423]\n[1368, 94, 4, 404, 2, 108, 9, 427, 47, 17, 1206, 9, 1480, 4, 470, 2]\n[887, 13, 32, 37, 1, 935, 3, 53, 1793, 23, 1, 209, 19]\n[71, 1, 21, 3, 174, 121, 3, 64, 2, 83, 54, 139, 119]\n[70, 11, 50, 145]\n[2, 71, 3, 1, 96, 16, 17, 12, 125, 20, 19, 4, 846, 17, 16, 1]\n[21, 3, 53, 2, 1132, 13, 5, 16, 271, 11, 1, 121, 2, 231, 9]\n[2310, 45, 1, 212, 4, 2311, 1, 142, 504, 1, 21, 3]\n[174, 25, 1794, 1616, 6, 1369, 49, 1616, 25, 92, 4, 1]\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(clines[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "12131\n7527\n"
     ]
    }
   ],
   "source": [
    "#获取最大词索引\n",
    "\n",
    "max_list = []\n",
    "for i in clines:\n",
    "    m = max(i)\n",
    "    max_list.append(m)\n",
    "print(len(max_list))\n",
    "print(max(max_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}